import java.io.BufferedReader;

import java.io.IOException;
import java.math.BigInteger;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Map;

import java.util.StringTokenizer;



/**

 * Author: Oscar Flores.

 * Description: Tokenizer class takes in a source file and breaks it down into tokens.

 *

 */

public class Tokenizer {

	/**

	 * reserved: a partial map of strings with keys being the reserved words in the Core language,

	 * and the values being the keys' respective numerical values as defined in the language.

	 */



private Map<String,String> reserved = new HashMap<String,String>();

/**

 * flag: a partial map of strings with the keys being on of he following "flags" to be used

 * during the tokenizing process and the values being the keys' respective numerical values as defined in the language.

 * keys: ; , = ! [ ] & | ( ) + - * < >

 */

private Map<String,String> flag = new HashMap<String,String>();	

/**

 * program: an arraylist used to keep ordered token numerical values defined in the Core language.

 */

private ArrayList<Integer> program = new ArrayList<Integer>();	

/**

 * symbols: a partial map of strings with keys being the symbols in the Core language,

 * and the values being the keys' respective numerical values as defined in the language.

 */

private Map<String,String> symbols = new HashMap<String,String>();	

/**

 * integer_and_id: a partial array of strings with keys being the String values of integers and ids in the source code,

 */

private ArrayList<String> integer_and_id = new ArrayList<String>();



/**

 * 

 * Module Name: Tokenizer.

 * Description:	Constructor for the tokenizer class.

 * Output Params: none.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public Tokenizer(BufferedReader source) throws IOException{

	Initialize();

	Tokenize_Source(source);

	

}



/**

 * 

 * Module Name: Tokenize.

 * Description:	takes in source code in the Core language and tokenize it line by line.

 * Input Params: BufferedReader source: the source code of the Core program.

 * Output Params: none.

 * Error Conditions Tested: source code file is not empty,

 * 							first token must be "program"

							second to last token must be "end"

							third to last must be ";".	

							source code doesn't contain "begin" keyword 

							source code must contain at least one "int" declaration

 * Error Messages Generated: missing (program|end|;).

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private void Tokenize_Source(BufferedReader source) throws IOException {

	String line ="";

	

	while((line = source.readLine()) != null){

		//System.out.println("ysy");

		Tokenize_line(line);

	}

	this.program.add(33);

	// add and EOF token to the program



}



/**

 * 

 * Module Name: Tokenize.

 * Description:	takes in one line source code in the Core language and tokenizes it.

 * Input Params: String line: one line of the source code of a Core program.

 * Output Params: none.

 * Error Conditions Tested: program contains an invalid character.

 * 							id length is > 10

 * 							missing whitespace between reserved words.

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private void Tokenize_line(String line) {

	// tokenize line by white space

	StringTokenizer no_white_spaces = new StringTokenizer(line, " \t");

	StringBuffer current_token = new StringBuffer ("");

	// look for tokens if while the tokenizer has tokens	

	while(no_white_spaces.hasMoreTokens())

	{

		// get next token

		String token = no_white_spaces.nextToken();	

		

		while(token.length() > 0)

		{

			int i= 0;

			//add character to current token until flag 

			current_token = new StringBuffer ("");

			boolean has_seen = false;

			

			for( i=0; i<token.length() && !(has_seen = is_flag(token.charAt(i)));i++)

			{

				current_token.append(token.charAt(i));

				if(is_reserved(current_token.toString()))

					break;

			}

			

			if((!has_seen && is_reserved(current_token.toString())) && token.length() > current_token.length())

			{

				i++;

				if(!is_flag(token.charAt(i))){

					System.out.println(token.charAt(i));

					System.out.println("ERROR missing white space in line: " + token.toString() + "\nafter token \"" + current_token +"\"");

					System.exit(1);

				}

			}

			// set the flag aside to add to program

			String flag = "";

			if(has_seen){

				flag = flag +token.charAt(i);

			}

			//update token-|current_token| 	

			//System.out.println(line);

			//System.out.println(token);

			//System.out.println("Current Token: " + current_token);

			//System.out.println("Flag: " + flag);

			if(token.length()==1){

				token = "";

			}

			if(has_seen && token.length() >0)

				token = token.substring(current_token.length()+1);

			else

				if(token.length() > current_token.length())

				token = token.substring(current_token.length());

				else

					token = "";

			

			//System.out.println(token);

			// check if is valid token if it's not empty

			if(current_token.length() >0)

			{

				// check if is valid token

				//System.out.println(current_token);

				

				if(is_reserved(current_token.toString()))

				{

					String value = reserved.get(current_token.toString());

					this.program.add(Integer.parseInt(value));

					reserved.put(current_token.toString(), value);

				}

				else if(is_valid_integer(current_token.toString()))

				{

					this.program.add(31);

					this.integer_and_id.add(current_token.toString());
					

				}

				else if(is_valid_id(current_token.toString()))

				{

					this.program.add(32);

					this.integer_and_id.add(current_token.toString());

				}

				else{

					System.out.println("ERROR invalid character in code, or invalid id name, or id length: " + current_token.toString());

					System.exit(1);

				}



			}

			// if the current token is empty do nothing with the empty token

			if(flag.length() >0)

			{	

			// check if flag could be extended flag	

				if(token.length() >0)

				{	

					Character p_flag = token.charAt(0);

					if(is_cont_flag(p_flag))

					{

					flag = flag+p_flag;

					token = token.substring(1);

					}

					//System.out.println("Flag:" + flag);

				}

				String value = this.symbols.get(flag);

				this.program.add(Integer.parseInt(value));

				this.symbols.put(flag, value);

				//System.out.println(program.toString());			

			}

			

		}	

			

	}

	

}



/**

 * 

 * Module Name: is_cont_flag.

 * Description:	takes one character and checks to see if the character is a valid continuation of a flag token

 * Input Params: Character p_flag: a possible second character of a flag token.

 * Output Params: true if p_flag = (=,&,|+), false otherwise.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private boolean is_cont_flag(Character p_flag) {

	if(p_flag == '='||p_flag == '&'||p_flag == '|')

		return true;

	return false;

}

/**

 * 

 * Module Name: is_valid_id.

 * Description:	takes a String and checks to see if it's a valid id token.

 * Input Params: String current_token: a possible id token.

 * Output Params: true if current_token is a valid id.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private boolean is_valid_id(String current_token) {
	if(current_token.length() <1){

		return false;	

	}
	
	Character letter;

	letter = current_token.charAt(0);
	

	if(!Character.isUpperCase(letter))

		return false;


	if(current_token.length() > 10)

		return false;

	if(current_token.length() >1){

		int c = 0;

		while(c < current_token.length()){
			letter = current_token.charAt(c);
			if(!Character.isUpperCase(letter)){

				break;			
			}
			c++;
		}	
		if(c==current_token.length()){
			return true;
		}
		else{
			while(c<current_token.length()){
				letter = current_token.charAt(c);
				if(!Character.isDigit(letter))
					return false;
				c++;
			}
		}

	}

	return true;

}



/**

 * 

 * Module Name: is_valid_integer.

 * Description:	takes a String and checks to see if it's a valid integer token.

 * Input Params: String current_token: a possible integer(no) token.

 * Output Params: true if current_token is a valid integer, false otherwise.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */



private boolean is_valid_integer(String current_token) {

	for(int i=0; i <current_token.length();i++){

		if(!Character.isDigit(current_token.charAt(i)))

			return false;	

	}



	return true;

}





/**

 * 

 * Module Name: is_reserved.

 * Description:	takes a String and checks to see if it's a reserved word in the Core language.

 * Input Params: String current_token: a possible reserved(no) token.

 * Output Params: true if current_token is a valid reserved word, false otherwise.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private boolean is_reserved(String current_token) {

	boolean res =this.reserved.containsKey(current_token);

	return res;

}



/**

 * 

 * Module Name: is_reserved.

 * Description:	takes a String and checks to see if it's a flag that ends a token.

 * Input Params: String current_token: a possible flag(no) token.

 * Output Params: true if current_token is a valid flag, false otherwise.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private boolean is_flag(char f) {

	String flag = "" + f;

	boolean is = this.flag.containsKey(flag);

	

	return is;

}



/**

 * 

 * Module Name: Initialize.

 * Description:	fills up reserved, symbols and flag to use in syntax checking.

 * Input Params: none.

 * Output Params: none.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

private void Initialize() {

	reserved.put("program", "1");

	reserved.put("begin", "2");

	reserved.put("end", "3");

	reserved.put("int", "4");

	reserved.put("if", "5");

	reserved.put("then", "6");

	reserved.put("else", "7");

	reserved.put("while", "8");

	reserved.put("loop", "9");

	reserved.put("read", "10");

	reserved.put("write", "11");

	

	symbols.put(";", "12");

	symbols.put(",", "13");

	symbols.put("=", "14");

	symbols.put("!", "15");

	symbols.put("[", "16");

	symbols.put("]", "17");

	symbols.put("&&", "18");

	symbols.put("||", "19");

	symbols.put("(", "20");

	symbols.put(")", "21");

	symbols.put("+", "22");

	symbols.put("-", "23");

	symbols.put("*", "24");

	symbols.put("!=", "25");

	symbols.put("==", "26");

	symbols.put("<", "27");

	symbols.put(">", "28");

	symbols.put("<=", "29");

	symbols.put(">=", "30");

	

	flag.put(";", "12");

	flag.put(",", "13");

	flag.put("=", "14");

	flag.put("!", "15");

	flag.put("[", "16");

	flag.put("]", "17");

	flag.put("&", "18");

	flag.put("|", "19");

	flag.put("(", "20");

	flag.put(")", "21");

	flag.put("+", "22");

	flag.put("-", "23");

	flag.put("*", "24");

	flag.put("<", "27");

	flag.put(">", "28");



	

	}

/**

 * 

 * Module Name: print.

 * Description:	prints all the token values currently in the Tokenizer one token per line.

 * Input Params: none.

 * Output Params: none.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public void print(){

	for(int i = 0; i <this.program.size(); i++){

		System.out.println(this.program.get(i));

	}
	
	//System.out.println(this.program.toString());

	//System.out.println(this.integer_and_id.toString());
	System.out.println(this.integer_and_id);
	

}

/**

 * 

 * Module Name: getToken.

 * Description:	returns an int with the value of the current token. Repeated calls to getToken() returns the same token.

 * Input Params: none.

 * Output Params: returns an int with the value of the current token. Repeated calls to getToken() returns the same token. Returns -1 if there are no more tokens.

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public int getToken(){
		if(this.program.size() > 0){
			return this.program.get(0);
			
			
		}
		return -1;
		


}

/**

 * 

 * Module Name: skipToken.

 * Description:	skips the current token.

 * Input Params: none.

 * Output Params: none

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public void skipToken(){
	
	if(this.program.size()>0){
		if(this.program.get(0) == 31 ||this.program.get(0) == 32)
			{
				this.integer_and_id.remove(0);
			}
		this.program.remove(0);		
	}


}

/**

 * 

 * Module Name: intVal.

 * Description:	returns the int value of the current token if it is an integer token, otherwise prints out an error and exits the program.

 * Input Params: none.

 * Output Params: returns the int value of the current token if it is an integer

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public String intVal(){

	if(this.program.get(0)== 31)

	{
		return this.integer_and_id.get(0);
	}

	else{

		System.out.println("ERROR: Current token is not an integer");

		System.exit(1);

		

	}
	return "-1";

}

/**

 * 

 * Module Name: idName.

 * Description:	returns the String value of the current token if it is an id token, otherwise prints out an error and exits the program.

 * Input Params: none.

 * Output Params: returns the int value of the current token if it is an integer

 * Error Conditions Tested:  None.	

 * Error Messages Generated: None.

 * Original Author: Oscar Flores.

 * Date of Installation: 10/08/2011.

 */

public String idName(){

	if(this.program.get(0)== 32)

	{

		return (this.integer_and_id.get(0));

	}

	else{

		System.out.println("ERROR: Current token is not an id");

		System.exit(1);

		

	}

	return null;

}



}


